{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一小节是在Tensorflow上编写的，为了调整超参数方便，将代码改为使用layer实现，具体如下。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting ./MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting ./MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./MNIST/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# 导入数据\n",
    "data_dir = './MNIST'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义数据\n",
    "x = tf.placeholder(tf.float32, [None, 784])   # 输入图片的大小，28x28=784\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])   # 输出0-9共10个数字\n",
    "learning_rate = tf.placeholder(tf.float32)    # 用于接收dropout操作的值，dropout为了防止过拟合\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "#-1代表先不考虑输入的图片例子多少这个维度，后面的1是channel的数量，因为我们输入的图片是黑白的，因此channel是1，例如如果是RGB图像，那么channel就是3\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 卷积层定义\n",
    "#函数参数中的filter_size是指卷积核的大小,step表示布长\n",
    "def conv_op(input_op, filter_size, channel_out, name):\n",
    "    h_conv1 = tf.layers.conv2d(input_op, channel_out, [filter_size,filter_size],\n",
    "                             padding='SAME',\n",
    "                             activation=tf.nn.relu,name=name)    \n",
    "    return h_conv1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 最大池化层\n",
    "def maxPool_op(input_op, filter_size, step, name):\n",
    "    h_pool1 = tf.layers.max_pooling2d(input_op, pool_size=[filter_size,filter_size],\n",
    "                        strides=[step, step], padding='VALID',name=name)\n",
    "    return h_pool1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ndef full_connection(input_op, channel_out, name):\\n    channel_in = input_op.get_shape()[-1].value\\n    with tf.name_scope(name) as scope:\\n        weight = tf.Variable(tf.truncated_normal([channel_in, channel_out],mean=0,\\n                                                  dtype=tf.float32, stddev=0.1),\\n                                                  collections=[tf.GraphKeys.GLOBAL_VARIABLES,'WEIGHTS'])\\n        #weight = tf.get_variable(shape=[channel_in, channel_out], dtype=tf.float32,\\n        #                         initializer=xavier_initializer_conv2d(), name=scope + 'weight')\\n        bias = tf.Variable(tf.constant(value=0.0, shape=[channel_out], dtype=tf.float32), name='bias')\\n        input_op_reshape = tf.reshape(input_op, [-1, 7 * 7 * 64])\\n        fc = tf.nn.relu(tf.matmul(input_op_reshape, weight) + bias)\\n        return fc\\n\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 全连接层\n",
    "'''\n",
    "def full_connection(input_op, channel_out, name):\n",
    "    channel_in = input_op.get_shape()[-1].value\n",
    "    with tf.name_scope(name) as scope:\n",
    "        weight = tf.Variable(tf.truncated_normal([channel_in, channel_out],mean=0,\n",
    "                                                  dtype=tf.float32, stddev=0.1),\n",
    "                                                  collections=[tf.GraphKeys.GLOBAL_VARIABLES,'WEIGHTS'])\n",
    "        #weight = tf.get_variable(shape=[channel_in, channel_out], dtype=tf.float32,\n",
    "        #                         initializer=xavier_initializer_conv2d(), name=scope + 'weight')\n",
    "        bias = tf.Variable(tf.constant(value=0.0, shape=[channel_out], dtype=tf.float32), name='bias')\n",
    "        input_op_reshape = tf.reshape(input_op, [-1, 7 * 7 * 64])\n",
    "        fc = tf.nn.relu(tf.matmul(input_op_reshape, weight) + bias)\n",
    "        return fc\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第一层卷积层，卷积核为5*5，深度为32，步长为1，输出为28*28*32\n",
    "conv1=conv_op(x_image,filter_size=5,channel_out=32,name='conv1')\n",
    "#第一个池化层，输出14*14*28\n",
    "pool1=maxPool_op(conv1,filter_size=2,step=2,name='pool1')\n",
    "#第二层卷积层，卷积核为5*5，深度为64，步长为1，输出为28*28*64\n",
    "conv2=conv_op(pool1,filter_size=5,channel_out=64,name='conv2')\n",
    "#第二个池化层，输出7*7*64\n",
    "pool2=maxPool_op(conv2,filter_size=2,step=2,name='pool2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.contrib.layers import flatten\n",
    "#全连接层，映射7*7*64特征图，映射为1024个特征\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = flatten(pool2)\n",
    "  h_fc1 = tf.layers.dense(h_pool2_flat, 1024, activation=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "#这里同上，需要注意的是，最后暂不需要使用激活函数\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.layers.dense(h_fc1_drop, 10, activation=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置正则化方法\n",
    "REGULARIZATION_RATE = 0.0001 # 比较合适的参数\n",
    "\n",
    "regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)  # 定义L2正则化损失函数\n",
    "#regularization = regularizer(weights1) + regularizer(weights2)  # 计算模型的正则化损失"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 1.840661, l2_loss: 0.080467, total loss: 1.921128\n",
      "0.57\n",
      "step 200, entropy loss: 0.734794, l2_loss: 0.080637, total loss: 0.815431\n",
      "0.78\n",
      "step 300, entropy loss: 0.508652, l2_loss: 0.080728, total loss: 0.589380\n",
      "0.87\n",
      "step 400, entropy loss: 0.440330, l2_loss: 0.080785, total loss: 0.521115\n",
      "0.9\n",
      "step 500, entropy loss: 0.465968, l2_loss: 0.080820, total loss: 0.546788\n",
      "0.89\n",
      "step 600, entropy loss: 0.297921, l2_loss: 0.080854, total loss: 0.378775\n",
      "0.93\n",
      "step 700, entropy loss: 0.182660, l2_loss: 0.080879, total loss: 0.263539\n",
      "0.96\n",
      "step 800, entropy loss: 0.226560, l2_loss: 0.080900, total loss: 0.307460\n",
      "0.94\n",
      "step 900, entropy loss: 0.116217, l2_loss: 0.080918, total loss: 0.197135\n",
      "0.97\n",
      "step 1000, entropy loss: 0.173367, l2_loss: 0.080939, total loss: 0.254305\n",
      "0.95\n",
      "0.934\n",
      "step 1100, entropy loss: 0.107668, l2_loss: 0.080954, total loss: 0.188622\n",
      "0.96\n",
      "step 1200, entropy loss: 0.108892, l2_loss: 0.080968, total loss: 0.189860\n",
      "0.93\n",
      "step 1300, entropy loss: 0.205523, l2_loss: 0.080981, total loss: 0.286504\n",
      "0.98\n",
      "step 1400, entropy loss: 0.194557, l2_loss: 0.080994, total loss: 0.275552\n",
      "0.97\n",
      "step 1500, entropy loss: 0.257998, l2_loss: 0.081000, total loss: 0.338998\n",
      "0.9\n",
      "step 1600, entropy loss: 0.097373, l2_loss: 0.081010, total loss: 0.178383\n",
      "0.98\n",
      "step 1700, entropy loss: 0.095919, l2_loss: 0.081020, total loss: 0.176938\n",
      "0.98\n",
      "step 1800, entropy loss: 0.191460, l2_loss: 0.081024, total loss: 0.272483\n",
      "0.96\n",
      "step 1900, entropy loss: 0.198772, l2_loss: 0.081031, total loss: 0.279803\n",
      "0.95\n",
      "step 2000, entropy loss: 0.207263, l2_loss: 0.081038, total loss: 0.288301\n",
      "0.95\n",
      "0.9568\n",
      "step 2100, entropy loss: 0.110423, l2_loss: 0.081044, total loss: 0.191466\n",
      "0.96\n",
      "step 2200, entropy loss: 0.180934, l2_loss: 0.081050, total loss: 0.261984\n",
      "0.95\n",
      "step 2300, entropy loss: 0.129383, l2_loss: 0.081054, total loss: 0.210436\n",
      "0.95\n",
      "step 2400, entropy loss: 0.109266, l2_loss: 0.081055, total loss: 0.190321\n",
      "0.97\n",
      "step 2500, entropy loss: 0.109986, l2_loss: 0.081057, total loss: 0.191043\n",
      "0.98\n",
      "step 2600, entropy loss: 0.049777, l2_loss: 0.081061, total loss: 0.130838\n",
      "1.0\n",
      "step 2700, entropy loss: 0.091289, l2_loss: 0.081063, total loss: 0.172352\n",
      "0.97\n",
      "step 2800, entropy loss: 0.173636, l2_loss: 0.081067, total loss: 0.254703\n",
      "0.96\n",
      "step 2900, entropy loss: 0.077945, l2_loss: 0.081068, total loss: 0.159013\n",
      "0.97\n",
      "step 3000, entropy loss: 0.082274, l2_loss: 0.081070, total loss: 0.163344\n",
      "0.98\n",
      "0.9669\n",
      "step 3100, entropy loss: 0.095772, l2_loss: 0.081071, total loss: 0.176843\n",
      "0.98\n",
      "step 3200, entropy loss: 0.087618, l2_loss: 0.081069, total loss: 0.168687\n",
      "0.99\n",
      "step 3300, entropy loss: 0.198064, l2_loss: 0.081070, total loss: 0.279134\n",
      "0.95\n",
      "step 3400, entropy loss: 0.106539, l2_loss: 0.081070, total loss: 0.187609\n",
      "0.98\n",
      "step 3500, entropy loss: 0.124284, l2_loss: 0.081068, total loss: 0.205352\n",
      "0.96\n",
      "step 3600, entropy loss: 0.061681, l2_loss: 0.081067, total loss: 0.142749\n",
      "1.0\n",
      "step 3700, entropy loss: 0.070378, l2_loss: 0.081065, total loss: 0.151443\n",
      "0.98\n",
      "step 3800, entropy loss: 0.214908, l2_loss: 0.081063, total loss: 0.295971\n",
      "0.94\n",
      "step 3900, entropy loss: 0.052038, l2_loss: 0.081059, total loss: 0.133098\n",
      "0.98\n",
      "step 4000, entropy loss: 0.070308, l2_loss: 0.081056, total loss: 0.151364\n",
      "0.99\n",
      "0.9717\n",
      "step 4100, entropy loss: 0.118724, l2_loss: 0.081055, total loss: 0.199779\n",
      "0.99\n",
      "step 4200, entropy loss: 0.052381, l2_loss: 0.081051, total loss: 0.133432\n",
      "0.98\n",
      "step 4300, entropy loss: 0.109384, l2_loss: 0.081050, total loss: 0.190434\n",
      "0.97\n",
      "step 4400, entropy loss: 0.090578, l2_loss: 0.081048, total loss: 0.171626\n",
      "0.98\n",
      "step 4500, entropy loss: 0.196057, l2_loss: 0.081043, total loss: 0.277100\n",
      "0.97\n",
      "step 4600, entropy loss: 0.030688, l2_loss: 0.081039, total loss: 0.111727\n",
      "0.97\n",
      "step 4700, entropy loss: 0.043534, l2_loss: 0.081036, total loss: 0.124570\n",
      "1.0\n",
      "step 4800, entropy loss: 0.054455, l2_loss: 0.081032, total loss: 0.135487\n",
      "1.0\n",
      "step 4900, entropy loss: 0.076947, l2_loss: 0.081028, total loss: 0.157975\n",
      "0.98\n",
      "step 5000, entropy loss: 0.054542, l2_loss: 0.081026, total loss: 0.135568\n",
      "0.98\n",
      "0.977\n",
      "step 5100, entropy loss: 0.057492, l2_loss: 0.081022, total loss: 0.138514\n",
      "0.98\n",
      "step 5200, entropy loss: 0.039550, l2_loss: 0.081018, total loss: 0.120568\n",
      "0.99\n",
      "step 5300, entropy loss: 0.064609, l2_loss: 0.081012, total loss: 0.145621\n",
      "0.96\n",
      "step 5400, entropy loss: 0.089577, l2_loss: 0.081008, total loss: 0.170585\n",
      "0.99\n",
      "step 5500, entropy loss: 0.061792, l2_loss: 0.081003, total loss: 0.142794\n",
      "0.96\n",
      "step 5600, entropy loss: 0.046460, l2_loss: 0.080999, total loss: 0.127459\n",
      "0.98\n",
      "step 5700, entropy loss: 0.144087, l2_loss: 0.080995, total loss: 0.225082\n",
      "0.98\n",
      "step 5800, entropy loss: 0.116287, l2_loss: 0.080990, total loss: 0.197276\n",
      "1.0\n",
      "step 5900, entropy loss: 0.062168, l2_loss: 0.080985, total loss: 0.143154\n",
      "0.99\n",
      "step 6000, entropy loss: 0.056647, l2_loss: 0.080980, total loss: 0.137627\n",
      "0.98\n",
      "0.9783\n",
      "step 6100, entropy loss: 0.056993, l2_loss: 0.080974, total loss: 0.137967\n",
      "0.99\n",
      "step 6200, entropy loss: 0.030714, l2_loss: 0.080969, total loss: 0.111682\n",
      "1.0\n",
      "step 6300, entropy loss: 0.010473, l2_loss: 0.080962, total loss: 0.091435\n",
      "0.99\n",
      "step 6400, entropy loss: 0.028236, l2_loss: 0.080957, total loss: 0.109194\n",
      "0.99\n",
      "step 6500, entropy loss: 0.025729, l2_loss: 0.080950, total loss: 0.106679\n",
      "0.98\n",
      "step 6600, entropy loss: 0.030856, l2_loss: 0.080945, total loss: 0.111801\n",
      "0.99\n",
      "step 6700, entropy loss: 0.135040, l2_loss: 0.080939, total loss: 0.215979\n",
      "0.96\n",
      "step 6800, entropy loss: 0.125772, l2_loss: 0.080933, total loss: 0.206704\n",
      "0.96\n",
      "step 6900, entropy loss: 0.044181, l2_loss: 0.080927, total loss: 0.125108\n",
      "0.97\n",
      "step 7000, entropy loss: 0.031386, l2_loss: 0.080920, total loss: 0.112306\n",
      "0.98\n",
      "0.9803\n",
      "step 7100, entropy loss: 0.046531, l2_loss: 0.080914, total loss: 0.127445\n",
      "0.99\n",
      "step 7200, entropy loss: 0.062767, l2_loss: 0.080908, total loss: 0.143675\n",
      "0.99\n",
      "step 7300, entropy loss: 0.038379, l2_loss: 0.080905, total loss: 0.119284\n",
      "0.99\n",
      "step 7400, entropy loss: 0.088725, l2_loss: 0.080897, total loss: 0.169622\n",
      "0.98\n",
      "step 7500, entropy loss: 0.081862, l2_loss: 0.080890, total loss: 0.162752\n",
      "0.98\n",
      "step 7600, entropy loss: 0.073651, l2_loss: 0.080884, total loss: 0.154535\n",
      "0.99\n",
      "step 7700, entropy loss: 0.097173, l2_loss: 0.080877, total loss: 0.178050\n",
      "0.98\n",
      "step 7800, entropy loss: 0.044475, l2_loss: 0.080870, total loss: 0.125346\n",
      "1.0\n",
      "step 7900, entropy loss: 0.016557, l2_loss: 0.080864, total loss: 0.097421\n",
      "1.0\n",
      "step 8000, entropy loss: 0.028215, l2_loss: 0.080856, total loss: 0.109071\n",
      "0.99\n",
      "0.9805\n",
      "step 8100, entropy loss: 0.047962, l2_loss: 0.080850, total loss: 0.128812\n",
      "0.98\n",
      "step 8200, entropy loss: 0.033914, l2_loss: 0.080843, total loss: 0.114757\n",
      "0.98\n",
      "step 8300, entropy loss: 0.131952, l2_loss: 0.080836, total loss: 0.212788\n",
      "0.96\n",
      "step 8400, entropy loss: 0.127357, l2_loss: 0.080827, total loss: 0.208183\n",
      "0.98\n",
      "step 8500, entropy loss: 0.038018, l2_loss: 0.080819, total loss: 0.118837\n",
      "0.99\n",
      "step 8600, entropy loss: 0.024061, l2_loss: 0.080811, total loss: 0.104873\n",
      "1.0\n",
      "step 8700, entropy loss: 0.075691, l2_loss: 0.080806, total loss: 0.156497\n",
      "0.99\n",
      "step 8800, entropy loss: 0.016437, l2_loss: 0.080797, total loss: 0.097234\n",
      "1.0\n",
      "step 8900, entropy loss: 0.101709, l2_loss: 0.080791, total loss: 0.182499\n",
      "0.96\n",
      "step 9000, entropy loss: 0.057965, l2_loss: 0.080783, total loss: 0.138748\n",
      "0.99\n",
      "0.9819\n",
      "step 9100, entropy loss: 0.047908, l2_loss: 0.080775, total loss: 0.128683\n",
      "0.99\n",
      "step 9200, entropy loss: 0.014077, l2_loss: 0.080767, total loss: 0.094844\n",
      "0.99\n",
      "step 9300, entropy loss: 0.024036, l2_loss: 0.080760, total loss: 0.104795\n",
      "0.98\n",
      "step 9400, entropy loss: 0.011912, l2_loss: 0.080753, total loss: 0.092665\n",
      "1.0\n",
      "step 9500, entropy loss: 0.081573, l2_loss: 0.080744, total loss: 0.162317\n",
      "0.98\n",
      "step 9600, entropy loss: 0.106740, l2_loss: 0.080738, total loss: 0.187478\n",
      "0.96\n",
      "step 9700, entropy loss: 0.076546, l2_loss: 0.080730, total loss: 0.157277\n",
      "0.99\n",
      "step 9800, entropy loss: 0.019649, l2_loss: 0.080721, total loss: 0.100370\n",
      "1.0\n",
      "step 9900, entropy loss: 0.012985, l2_loss: 0.080713, total loss: 0.093698\n",
      "1.0\n",
      "step 10000, entropy loss: 0.031015, l2_loss: 0.080706, total loss: 0.111721\n",
      "0.99\n",
      "0.9837\n"
     ]
    }
   ],
   "source": [
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "regularization=0.0\n",
    "for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
    "    regularization=regularization+regularizer(w)\n",
    "l2_loss=regularization\n",
    "#l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection('WEIGHTS')] )\n",
    "#total_loss = cross_entropy + 7e-5*l2_loss\n",
    "total_loss = cross_entropy + l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(10000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "心得与小结:\n",
    "本小节功能与第一小节一样，这里改为使用layer实现，为了后续超参数调整方便。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
